package org.killgfw.gfwresources

import java.net.{URL, URLDecoder, InetAddress}
import java.io.{FileInputStream, FileOutputStream, File}
import org.apache.commons.io.{IOUtils, FileUtils}
import io.Source
import org.apache.commons.codec.binary.Base64

/**
 *
 *
 * @author <a href="mailto:hyysguyang@gamil.com">Young Gu</a>
 * @author <a href="mailto:Young.Gu@lifecosys.com">Young Gu</a>
 * @version 1.0 1/10/13 11:40 PM
 */

object ResourceGenerator {
  def main(args: Array[String]) {
    ResourceGenerator().generate
  }

  def apply()=new ResourceGenerator

}

class ResourceGenerator {
  def extractHostAndPort(uri: String) = {

    val url = if (uri.startsWith("http"))
      new URL(uri)
    else
      new URL("http://" + uri)
    (url.getHost, Some(url.getPort).filter(_ > 0).getOrElse(80))
  }


  def getGFWListContent: String = new String(Base64.decodeBase64(IOUtils.toString(new FileInputStream("gfwlist.txt"))))

  def generate {
    val list = Source.fromString(getGFWListContent).getLines().filterNot {
      line => line.startsWith("!") || line.startsWith("@@") || line.startsWith("/")
    }.toList

    def extractHost(line: String) = {
      val url = if (line.indexOf('*') > 0) {
        val extractFirstPart = line.replace( """\*+""", "*").replaceFirst( """\*+""", "").split('*')
        extractFirstPart(0)
      } else line
      val host = if (url.indexOf('?') > 0) url.substring(0, url.indexOf('?')) else url
      try {
        extractHostAndPort(host)._1
      }
      catch {
        case _ => ""
      }
    }
    def reachable(host: String) = {
      try {
        println("Checking if host cab be reachable:\t" + host)
        InetAddress.getByName(host).isReachable(1000)
      }
      catch {
        case _ => true
      }
    }

    def validHost(host: String) = {
      try {
        println("Checking host name:\t" + host)
        InetAddress.getByName(host) != null
      }
      catch {
        case _ => false
      }
    }


    val hosts = list.par.map {
      case line if (line.startsWith("||")) => extractHost(line.substring(2))
      case line if (line.startsWith("|") || line.startsWith(".") || line.startsWith("*")) => extractHost(line.substring(1))
      case line => extractHost(line)
    }.toSet.filter(validHost _).filterNot(reachable _)


    def isLetter(ch: String) = !ch.exists(_ > 256)
    val sensitiveWords = list.par.flatMap {
      case line if (line.indexOf('*', 1) > 0) => {
        val decode = URLDecoder.decode(line, "UTF-8")
        decode.substring(decode.indexOf('*', 1)).split('*').filterNot(isLetter _)
      }
      case line => Nil
    }.toSet



    println("Writing host list: %s and sensitive words: %s to file......".format(hosts.size, sensitiveWords.size))

    val generatedFile: File = new File("generated")
    FileUtils.deleteQuietly(generatedFile)
    generatedFile.mkdir()
    val hostListFile = new File(generatedFile.getAbsolutePath +"/gfw-host-list-generated.txt")
    val sensitiveFile = new File(generatedFile.getAbsolutePath +"/gfw-sensitive-words-generated.txt")
    import scala.collection.JavaConverters._
    IOUtils.writeLines(hosts.seq.asJavaCollection, null, new FileOutputStream(hostListFile))
    IOUtils.writeLines(sensitiveWords.seq.asJavaCollection, null, new FileOutputStream(sensitiveFile))

    println("Generate completed....")
    println("Host list file:  "+hostListFile)
    println("Sensitive words list file:  "+sensitiveFile)
  }


}
